package tools.crawler.ctrip;

import java.sql.PreparedStatement;
import java.util.ArrayList;
import java.util.List;
import java.util.UUID;

import org.junit.Test;

import com.easywan.util.DBUtils;
import com.easywan.util.DateUtils;
import com.easywan.util.ImageUtils;
import com.easywan.util.StringsUtils;
import com.easywan.util.Utils;

import tools.crawler.utils.HttpRequest;

public class YouCtrapCom {
	@Test
	public void start(){
		//step1	获取所有主页资源
		String indexUrl = "http://you.ctrip.com/travels/Shenzhen26/t3-p[1].html";
		List<String> indexUrls =  StringsUtils.urlToStrings(indexUrl, 1, 3, 1, "[1]");
		System.out.println(indexUrls);
		System.out.println(indexUrls.size());
		List<String> contentUrls = new ArrayList<>();
		for(String url : indexUrls){
			System.out.println("url start------獲取文章頁URL"+url);
			String indexHtml =  HttpRequest.get(url).body();
			indexHtml = StringsUtils.findMid(indexHtml, "<div class=\"practicaltab\">", "<!--国家游记列表等用的翻页形式-->");
			//System.out.println(indexHtml);
			String regex = "href=\"(.+?)\"";
			List<String> urls = StringsUtils.mathesAndNot(indexHtml, regex,"t3");
			System.out.println(urls);
			contentUrls.addAll(urls);
			System.out.println("url end------獲取文章頁URL");
			//break;
		}
		System.out.println(contentUrls);
		Utils.listNotRe(contentUrls);
		//http://you.ctrip.com/
		//if(true) return ;
		//step2	获取所有文章页面
		//step3	根据文章页面获取数据
		for(String testUrl:contentUrls){
		//String  testUrl = "/travels/shenzhen26/2638847.html";
			try{
				System.out.println("content start------獲取文章頁內容"+testUrl);
				String htmlContent = HttpRequest.get("http://you.ctrip.com/"+testUrl).body();
				String realContent = StringsUtils.findMid(htmlContent, "<!-- 游记正文 -->","<!-- 游记正文 end -->");
				String titleStart = "<title>";
				String titleEnd = "</title>";
				String title = StringsUtils.findMid(htmlContent,titleStart,titleEnd);
				System.out.println(title);
				
				String keywordContent = StringsUtils.findMid(realContent,"<dl>","</dl>");
				List<String> keywords = StringsUtils.mathes(keywordContent, "<a.*>(.+?)</a>");
				System.out.println(keywords);
				
				String author = StringsUtils.findMid(htmlContent, "<div class=\"ctd_head_right\">", "<!-- 徽章控件 -->");
				//System.out.println(author);
				author = StringsUtils.mathes(author,"title=\"(.+?)\"").get(0);
				System.out.println(author);
				
				String source = testUrl;
				System.out.println(source);
				
				//採集所有圖片
				String regex = "<img class=\'lary_1\' data-original=\'(.+?)\'";
				String path = "/upload/images/";
				String now = DateUtils.dateToPath();
				List<String> imageUrls = StringsUtils.mathes(realContent, regex);
				if(Utils.listIsEmpty(imageUrls)){
					imageUrls = StringsUtils.mathes(realContent, "<img data-original=\"(.+?)\"");
				}
				System.out.println(imageUrls);
				for(String imageUrl:imageUrls){
					String imagePath = path+now+UUID.randomUUID().toString()+".jpg";
					try{
					if(ImageUtils.save(imageUrl,imagePath,600) ){
						String sql3 = "insert into images(name,source) values(?,?)";
						PreparedStatement imagePstmt = DBUtils.getConnection().prepareStatement(sql3);
						imagePstmt.setString(1, imagePath);
						imagePstmt.setString(2, imageUrl);
						imagePstmt.executeUpdate();
						imagePstmt.close();
						realContent.replaceAll(imagePath, imageUrl);
						imagePstmt.close();
						System.out.println(imagePath+":"+ imageUrl);
					}
					}catch(Exception e){
						e.printStackTrace();
					}
				}
				String cover = Utils.listGetOne(imageUrls);
				cover = cover == null?"":cover;
				System.out.println(cover);
				//realContent.replaceAll(imagePath, imageUrl);
				String bief = StringsUtils.html2Text(realContent).substring(0, 50);
				System.out.println(bief);
				//step4	存储
				String sql3 = "insert into cj_travelogue(title,keyword,author,source,cover,bief,content) values(?,?,?,?,?,?,?)";
				try{
					PreparedStatement imagePstmt = DBUtils.getConnection().prepareStatement(sql3);
					imagePstmt.setString(1, title);
					imagePstmt.setString(2, StringsUtils.listToString(keywords, ','));
					imagePstmt.setString(3, author);
					imagePstmt.setString(4, source);
					imagePstmt.setString(5, cover);
					imagePstmt.setString(6, bief);
					imagePstmt.setString(7, realContent);
					imagePstmt.executeUpdate();
					imagePstmt.close();
				}catch(Exception e){
					e.printStackTrace();
				}
			}catch(Exception f){
				f.printStackTrace();
			}
		}
	}
}
